package com.qianxun.spider;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.qianxun.spider.bean.SpiderBean;
import com.qianxun.spider.builder.SpiderBeanBuilder;
import com.qianxun.spider.http.DefaultHttpProxyPool;
import com.qianxun.spider.http.HttpProxy;
import com.qianxun.spider.http.HttpProxyPool;
import com.qianxun.spider.util.ObjectUtil;
import com.qianxun.spider.util.scheduler.SchedulerUtil;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;

import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * @author: huwei
 * @date: 2020/7/15 15:13
 * @version: 1.0.0
 */
@Slf4j
public class ProxyIpSpiderDemo {

    private DefaultHttpProxyPool pool = new DefaultHttpProxyPool(10 ,false ,false);
    /**
     * 代理ip爬虫运行状态
     */
    private AtomicBoolean spiderRunStatus = new AtomicBoolean(false);

    public void startSpider() {
        if(isActive()){
            return;
        }
        this.spiderRunStatus.set(true);
        try {
            SchedulerUtil.schedule(this::innerStartSpider);
        }catch (Throwable e){
            new Thread(this::innerStartSpider).start();
        }
    }

    public void stopSpider() {
        this.spiderRunStatus.set(false);
    }

    public boolean isActive() {
        return this.spiderRunStatus.get();
    }

    public HttpProxyPool getHttpProxyPool(int retry , boolean localRetry , boolean localPriority) {
        return new HttpProxyPool() {
            @Override
            public HttpProxy next() {
                if(isEmpty()){
                    //如果代理池数据为空，通知管理员
                    log.info("代理IP池数量为空" ,"代理IP池数量为空 ,代理爬虫状态：" + (isActive() ? "运行中" : "挂掉"));
                }
                return pool.next();
            }
            @Override
            public int retry() {
                return retry;
            }
            @Override
            public boolean localRetry() {
                return localRetry;
            }

            @Override
            public boolean localPriority() {
                return localPriority;
            }

            @Override
            public void makeFailure(HttpProxy httpProxy) {
                pool.makeFailure(httpProxy);
            }

            @Override
            public boolean isEmpty() {
                return pool.isEmpty();
            }

            @Override
            public int size() {
                return pool.size();
            }
        };
    }

    private void innerStartSpider() {
        System.out.println("---代理IP爬虫启动，准备爬取...");
        //数据爬取失败次数
        int failedCount = 0;
        //失败宽限次数，，当达到100次以上时，通知管理员，规避代理ip不可用造成的业务异常
        final int failedGrace = 100;
        while (this.spiderRunStatus.get()){
            try {
                long sleepTimeMillis;
                ProxyIpInfoBean proxyIpInfoBean = spiderProxyIp();
                if (proxyIpInfoBean == null) {
                    sleepTimeMillis = TimeUnit.SECONDS.toMillis(5);
                    if(++failedCount > failedGrace){
                        failedCount = 0;
                        log.error("代理IP爬取失败");
                        System.err.println("代理IP爬取失败");
                    }
                }else{
                    sleepTimeMillis = sleepTimeMillis(proxyIpInfoBean.getUpdateTimeMillis());
                    //IP过滤
                    //测试url地址
                    final String testUrl = "http://www.baidu.com";
                    //最大延迟容忍时间
                    final int timeout = 5000;

                    //废弃代理IP清除标志,用于清除上一次存入的代理IP，因为其多数已不可用
                    AtomicBoolean clearBeforeFlag = new AtomicBoolean(false);

                    proxyIpInfoBean.getIpList().forEach(proxyIpBean -> {
                        try {
                            int statusCode = Jsoup.connect(testUrl).timeout(timeout).proxy(proxyIpBean.getHost() ,proxyIpBean.getPort()).execute().statusCode();
                            if(statusCode >= 200 && statusCode < 300){
                                if(!clearBeforeFlag.get()){
                                    System.err.println("--- older proxy ip was clear");
                                    clearBeforeFlag.set(true);
                                    this.pool.clear();
                                }
                                this.pool.addLast(new HttpProxy(proxyIpBean.getHost() ,proxyIpBean.getPort()) ,sleepTimeMillis);
                                System.err.println("--- spider available proxy ip: " + proxyIpBean.getHost());
                            }
                        } catch (IOException e) {
                            //请求异常，该代理IP不可用
                        }
                    });
                }
                log.info("--代理IP爬虫即将进入休眠时间，预计休眠：" + sleepTimeMillis + "毫秒");
                System.out.println("--代理IP爬虫即将进入休眠时间，预计休眠：" + sleepTimeMillis + "毫秒");
                //爬取失败，休眠5秒重新爬取
                synchronized (Thread.currentThread()){
                    Thread.currentThread().wait(sleepTimeMillis);
                }
            } catch (InterruptedException e){
                //线程中断异常
                break;
            }
        }
        System.err.println("---- 代理IP爬虫已退出服务");
        this.spiderRunStatus.set(false);
    }

    private long sleepTimeMillis(long updateTimeMillis){
        //proxy list 15分钟更新一次，目前以该网站ip作为主要ip进行更新
        return TimeUnit.MINUTES.toMillis(15) - (System.currentTimeMillis() - updateTimeMillis);
    }

    private ProxyIpInfoBean spiderProxyIp(){
        // proxy list 网站 15分钟更新一次
        SpiderBean spiderBean = SpiderBeanBuilder.newly()
                .fieldBuilder().fieldName("ipInfoJsonList").collection(true)
                .selector("table.highlight.tab-size.js-file-line-container > tbody > tr > td:nth-child(2)").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("updateTime").selector("body > div.application-main > div > main > div.container-xl.clearfix.new-discussion-timeline.px-3.px-md-4.px-lg-5 > div > div.Box.d-flex.flex-column.flex-shrink-0.mb-3 > div.Box-header.Box-header--blue.Details.js-details-container > div > div.flex-1.d-flex.flex-items-center.ml-3.min-width-0 > div > span:nth-child(2) > a")
                .valueKey("text").buildFieldAndAdd().build();
        Object o = Spider.get(spiderBean,"https://github.com/fate0/proxylist/blob/master/proxy.list");
        if(o != null){
            try {
                Map<String, Object> objectMap = ((Map<String, Object>) o);
                List<Object> ipInfoList = (List<Object>) objectMap.get("ipInfoJsonList");
                List<ProxyIpBean> proxyIpBeans = new ArrayList<>(ipInfoList.size());
                ipInfoList.forEach(info -> {
                    JSONObject jsonObject = JSON.parseObject(info.toString());
                    ProxyIpBean proxyIpBean = new ProxyIpBean();
                    proxyIpBean.setHost(jsonObject.getString("host")).setPort(jsonObject.getInteger("port"))
                            .setResponseTimeMillis((long) (jsonObject.getDouble("response_time") * 1000));
                    proxyIpBeans.add(proxyIpBean);
                });
                Collections.sort(proxyIpBeans, (o1, o2) -> o1.responseTimeMillis > o2.responseTimeMillis ? 1 : (o1.responseTimeMillis == o2.responseTimeMillis ? 0 : -1));

                String updateTimeStr = objectMap.get("updateTime").toString();
                long updateTimeMillis = ObjectUtil.isBlank(updateTimeStr) ? System.currentTimeMillis() : new Date(updateTimeStr).getTime();

                ProxyIpInfoBean proxyIpInfoBean = new ProxyIpInfoBean();
                proxyIpInfoBean.setUpdateTimeMillis(updateTimeMillis).setIpList(proxyIpBeans);
                return proxyIpInfoBean;
            }catch (Throwable throwable){
                //防止目标网站数据变更导致数据解析错误，从而导致退出ip爬取
                log.error("爬出代理IP出现异常，目标地址：https://github.com/fate0/proxylist/blob/master/proxy.list",throwable);
            }
        }
        return null;
    }

    @Data
    @Accessors(chain = true)
    private static class ProxyIpInfoBean {
        private long updateTimeMillis;
        private List<ProxyIpBean> ipList;
    }

    @Data
    @Accessors(chain = true)
    private static class ProxyIpBean {
        private String host;
        private int port;
        private long responseTimeMillis;
    }
}
